{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Exathlon (VLDB21) datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from typing import List, Optional\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "from config import data_raw_folder, data_processed_folder\n",
    "from timeeval import Datasets\n",
    "from timeeval.datasets import DatasetAnalyzer, DatasetRecord\n",
    "from IPython.display import display, Markdown\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (20, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def find_datasets(folder):\n",
    "    if not isinstance(folder, Path):\n",
    "        folder = Path(folder)\n",
    "    return sorted([f for d in source_folder.iterdir() if d.is_dir() for f in d.iterdir() if f.is_file()])\n",
    "\n",
    "def plot_dataset(f, start_column_idx=0, end_column_idx=10):\n",
    "    name = f.stem\n",
    "    column_idxs = [0] + list(range(start_column_idx+1, end_column_idx))\n",
    "    data = pd.read_csv(f).iloc[:, column_idxs].set_index(\"t\")\n",
    "    data.plot()\n",
    "    \n",
    "    # add anomaly labels\n",
    "    try:\n",
    "        anomalies = df_labels.loc[name]\n",
    "    except KeyError:\n",
    "        anomalies = pd.DataFrame([], columns=[\"begin\", \"end\"])\n",
    "    ax = plt.gca()\n",
    "    for name, row in anomalies.iterrows():\n",
    "        begin = row[\"begin\"].astype(np.int_)\n",
    "        end = row[\"end\"].astype(np.int_)\n",
    "        ax.add_patch(matplotlib.patches.Rectangle(\n",
    "            (begin, data.min().min()),\n",
    "            end-begin,\n",
    "            data.max().max()-data.min().min(),\n",
    "            color=\"yellow\", alpha=0.75\n",
    "        ))\n",
    "    plt.legend()\n",
    "    plt.title(name)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking for source datasets in /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw and\n",
      "saving processed datasets in /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed\n"
     ]
    }
   ],
   "source": [
    "dataset_collection_name = \"Exathlon\"\n",
    "source_folder = Path(data_raw_folder) / \"exathlon\" / \"data\" / \"raw\"\n",
    "target_folder = Path(data_processed_folder)\n",
    "\n",
    "print(f\"Looking for source datasets in {Path(source_folder).absolute()} and\\nsaving processed datasets in {Path(target_folder).absolute()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created directories /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon\n"
     ]
    }
   ],
   "source": [
    "# shared by all datasets\n",
    "dataset_type = \"real\"\n",
    "input_type = \"multivariate\"\n",
    "datetime_index = True\n",
    "split_at = None\n",
    "\n",
    "# create target directory\n",
    "dataset_subfolder = Path(input_type) / dataset_collection_name\n",
    "target_subfolder = target_folder / dataset_subfolder\n",
    "target_subfolder.mkdir(parents=True, exist_ok=True)\n",
    "print(f\"Created directories {target_subfolder}\")\n",
    "\n",
    "dm = Datasets(target_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>begin</th>\n",
       "      <th>end</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trace</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2_1_100000_60</th>\n",
       "      <td>1.527535e+09</td>\n",
       "      <td>1.527536e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2_1_100000_60</th>\n",
       "      <td>1.527550e+09</td>\n",
       "      <td>1.527551e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2_1_100000_60</th>\n",
       "      <td>1.527564e+09</td>\n",
       "      <td>1.527565e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_100000_61</th>\n",
       "      <td>1.527276e+09</td>\n",
       "      <td>1.527278e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_100000_61</th>\n",
       "      <td>1.527291e+09</td>\n",
       "      <td>1.527293e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_5_1000000_90</th>\n",
       "      <td>1.528967e+09</td>\n",
       "      <td>1.528967e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_4_1000000_82</th>\n",
       "      <td>1.528983e+09</td>\n",
       "      <td>1.528983e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_4_1000000_82</th>\n",
       "      <td>1.528987e+09</td>\n",
       "      <td>1.528987e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_5_1000000_92</th>\n",
       "      <td>1.528967e+09</td>\n",
       "      <td>1.528967e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6_5_1000000_93</th>\n",
       "      <td>1.528967e+09</td>\n",
       "      <td>1.528967e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>109 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       begin           end\n",
       "trace                                     \n",
       "2_1_100000_60   1.527535e+09  1.527536e+09\n",
       "2_1_100000_60   1.527550e+09  1.527551e+09\n",
       "2_1_100000_60   1.527564e+09  1.527565e+09\n",
       "4_1_100000_61   1.527276e+09  1.527278e+09\n",
       "4_1_100000_61   1.527291e+09  1.527293e+09\n",
       "...                      ...           ...\n",
       "4_5_1000000_90  1.528967e+09  1.528967e+09\n",
       "5_4_1000000_82  1.528983e+09  1.528983e+09\n",
       "5_4_1000000_82  1.528987e+09  1.528987e+09\n",
       "5_5_1000000_92  1.528967e+09  1.528967e+09\n",
       "6_5_1000000_93  1.528967e+09  1.528967e+09\n",
       "\n",
       "[109 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load ground truth\n",
    "df_labels = pd.read_csv(source_folder / \"ground_truth.csv\")\n",
    "\n",
    "# we use extended end marker for anomalies\n",
    "df_labels[\"extended_effect_end\"].fillna(df_labels[\"root_cause_end\"], axis=0, inplace=True)\n",
    "df_labels.drop(columns=[\"root_cause_end\", \"trace_type\", \"anomaly_type\", \"anomaly_details\"], inplace=True)\n",
    "df_labels.columns = [\"trace\", \"begin\", \"end\"]\n",
    "df_labels.set_index(\"trace\", inplace=True)\n",
    "df_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>tpe</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>app</th>\n",
       "      <th>rate</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>10000</th>\n",
       "      <td>1_0_10000_17</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>1_0_100000_15</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>1_0_100000_16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>1_2_100000_68</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500000</th>\n",
       "      <td>1_0_500000_18</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">10</th>\n",
       "      <th>1000000</th>\n",
       "      <td>10_2_1000000_67</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000000</th>\n",
       "      <td>10_3_1000000_75</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000000</th>\n",
       "      <td>10_4_1000000_79</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000000</th>\n",
       "      <td>10_5_1000000_85</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1200000</th>\n",
       "      <td>10_0_1200000_7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    filename tpe\n",
       "app rate                        \n",
       "1   10000       1_0_10000_17   0\n",
       "    100000     1_0_100000_15   0\n",
       "    100000     1_0_100000_16   0\n",
       "    100000     1_2_100000_68   2\n",
       "    500000     1_0_500000_18   0\n",
       "...                      ...  ..\n",
       "10  1000000  10_2_1000000_67   2\n",
       "    1000000  10_3_1000000_75   3\n",
       "    1000000  10_4_1000000_79   4\n",
       "    1000000  10_5_1000000_85   5\n",
       "    1200000   10_0_1200000_7   0\n",
       "\n",
       "[93 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load datasets and their metadata\n",
    "datasets = find_datasets(source_folder)\n",
    "datasets = dict((d.stem, d) for d in datasets)\n",
    "\n",
    "df_datasets = pd.DataFrame({\"filename\": list(datasets.keys())}, columns=[\"filename\", \"app\", \"tpe\", \"rate\"])\n",
    "df_datasets.iloc[:, 1:] = df_datasets[\"filename\"].str.split(\"_\", expand=True).iloc[:, :3]\n",
    "df_datasets.loc[:, [\"app\", \"tpe\", \"rate\"]] = df_datasets[[\"app\", \"tpe\", \"rate\"]].astype(np.int_)\n",
    "df_datasets = df_datasets.set_index([\"app\", \"rate\"]).sort_index()\n",
    "df_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define preprocessing methods\n",
    "def preprocess(dataset_name: Path, corr_threshold: float = 0.95, log_prefix: str = \"  \", preselected_columns: Optional[List[str]] = None):\n",
    "    exclude_columns = [\"timestamp\", \"is_anomaly\"]\n",
    "    print(f\"{log_prefix}reading dataset {dataset_name.name}\")\n",
    "    df = pd.read_csv(dataset_name)\n",
    "    s_index = pd.Index(pd.to_datetime(df[\"t\"], unit=\"s\"), name=\"timestamp\")\n",
    "    df.drop(columns=[\"t\"], inplace=True)\n",
    "    \n",
    "    print(f\"{log_prefix}adding timestamp index\")\n",
    "    df.index = s_index\n",
    "        \n",
    "    # add labels\n",
    "    print(f\"{log_prefix}adding anomaly labels\")\n",
    "    df[\"is_anomaly\"] = 0\n",
    "    try:\n",
    "        anomalies = df_labels.loc[dataset_name.stem:dataset_name.stem]\n",
    "    except KeyError:\n",
    "        anomalies = pd.DataFrame(columns=df_labels.columns, index=pd.Index([], name=\"trace\"))\n",
    "    for i, anomaly in anomalies.iterrows():\n",
    "        begin = pd.to_datetime(anomaly[\"begin\"], unit=\"s\")\n",
    "        end = pd.to_datetime(anomaly[\"end\"], unit=\"s\")\n",
    "        df.loc[begin:end, \"is_anomaly\"] = 1\n",
    "    \n",
    "    if preselected_columns is not None:\n",
    "        # use columns from test dataset (preselected)\n",
    "        print(f\"{log_prefix}reducing columns\")\n",
    "        columns = set(list(preselected_columns) + exclude_columns)\n",
    "        unavailable_columns = columns - set(df.columns)\n",
    "        if len(unavailable_columns) > 0:\n",
    "            warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
    "        df = df[[c for c in columns if c not in unavailable_columns]]\n",
    "    else:\n",
    "        # columns with no real value\n",
    "        print(f\"{log_prefix}removing constant and empty columns\")\n",
    "        to_drop = [c for c in df.columns if c not in exclude_columns and (np.all(df[c].isin([0, -1])) or df[c].unique().shape[0] == 1)]\n",
    "        df.drop(columns=to_drop, inplace=True)\n",
    "\n",
    "        # only select correlated values\n",
    "        print(f\"{log_prefix}selecting with label correlated columns\")\n",
    "        s_corr = df.drop(columns=[\"is_anomaly\"]).corrwith(df[\"is_anomaly\"], drop=True).abs()\n",
    "        cols = list(s_corr[s_corr > 0.5].index.values)\n",
    "        df = df[cols + [\"is_anomaly\"]]\n",
    "\n",
    "        # drop highly correlated columns\n",
    "        print(f\"{log_prefix}removing highly correlated columns\")\n",
    "        df_corr = df.drop(columns=[\"is_anomaly\"]).corr().abs()\n",
    "        df_corr = df_corr.where(np.triu(np.ones(df_corr.shape), k=1).astype(np.bool_))\n",
    "        to_drop = [c for c in df_corr.columns if any(df_corr[c] > corr_threshold)]\n",
    "        df.drop(columns=to_drop, inplace=True)\n",
    "        \n",
    "    # make timestamp index a column\n",
    "    df.reset_index(drop=False, inplace=True)\n",
    "    print(f\"{log_prefix}final dataset shape: {df.shape}\")\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def transform_and_record(s_dataset: pd.Series,\n",
    "                         s_partner: Optional[pd.Series] = None,\n",
    "                         train_is_normal: bool = False,\n",
    "                         train_type: str = \"unsupervised\"):\n",
    "    if s_partner is None:\n",
    "        train = False\n",
    "        dataset_name = s_dataset[\"filename\"]\n",
    "    else:\n",
    "        train = True\n",
    "        dataset_name = s_dataset[\"filename\"] + \"-\" + s_partner[\"filename\"].split(\"_\")[-1]\n",
    "\n",
    "    test_filename = f\"{dataset_name}.test.csv\"\n",
    "    test_path = dataset_subfolder / test_filename\n",
    "    target_test_filepath = target_subfolder / test_filename\n",
    "    target_meta_filepath = target_test_filepath.parent / f\"{dataset_name}.{Datasets.METADATA_FILENAME_PREFIX}\"\n",
    "    \n",
    "    if train:\n",
    "        train_filename = f\"{dataset_name}.train.csv\"\n",
    "        train_path = dataset_subfolder / train_filename\n",
    "        target_train_filepath = target_subfolder / train_filename\n",
    "    else:\n",
    "        train_path = \"\"\n",
    "\n",
    "    # Prepare test dataset\n",
    "    print(\"  Preparing test dataset\")\n",
    "    f = datasets[s_dataset[\"filename\"]]\n",
    "    df_test = preprocess(f, log_prefix=\"  > \")\n",
    "    test_columns = df_test.columns\n",
    "    if set(test_columns) == set([\"timestamp\", \"is_anomaly\"]):\n",
    "        warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n",
    "        return\n",
    "    df_test.to_csv(target_test_filepath, index=False)\n",
    "\n",
    "    print(\"  Analyzing test metadata\")\n",
    "    da = DatasetAnalyzer((dataset_collection_name, dataset_name), is_train=False, df=df_test)\n",
    "    da.save_to_json(target_meta_filepath, overwrite=True)\n",
    "    meta = da.metadata\n",
    "\n",
    "    # Prepare train dataset\n",
    "    if train:\n",
    "        print(\"  Preparing train dataset\")\n",
    "        f = datasets[s_partner[\"filename\"]]\n",
    "        df_train = preprocess(f, log_prefix=\"  > \", preselected_columns=test_columns)\n",
    "        df_train.to_csv(target_train_filepath, index=False)\n",
    "\n",
    "        print(\"  Analyzing train metadata\")\n",
    "        DatasetAnalyzer((dataset_collection_name, dataset_name), is_train=True, df=df_train)\\\n",
    "            .save_to_json(target_meta_filepath, overwrite=False)\n",
    "\n",
    "    dm.add_dataset(DatasetRecord(\n",
    "        collection_name=dataset_collection_name,\n",
    "        dataset_name=dataset_name,\n",
    "        train_path=train_path,\n",
    "        test_path=test_path,\n",
    "        dataset_type=dataset_type,\n",
    "        datetime_index=datetime_index,\n",
    "        split_at=split_at,\n",
    "        train_type=train_type,\n",
    "        train_is_normal=train_is_normal,\n",
    "        input_type=input_type,\n",
    "        length=meta.length,\n",
    "        dimensions=meta.dimensions,\n",
    "        contamination=meta.contamination,\n",
    "        num_anomalies=meta.num_anomalies,\n",
    "        min_anomaly_length=meta.anomaly_length.min,\n",
    "        median_anomaly_length=meta.anomaly_length.median,\n",
    "        max_anomaly_length=meta.anomaly_length.max,\n",
    "        mean=meta.mean,\n",
    "        stddev=meta.stddev,\n",
    "        trend=meta.trend,\n",
    "        stationarity=meta.get_stationarity_name(),\n",
    "        period_size=np.nan\n",
    "    ))\n",
    "    print(f\"Processed dataset {dataset_name}: {datasets[s_dataset['filename']]} -> {target_test_filepath}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "## Processing 1_2_100000_68-15\n",
      "  1_2_100000_68-15 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 1_2_100000_68.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (2936, 46)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 1_0_100000_15.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (2682, 45)\n",
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_1_20_10000_100000_batch076_1__StreamingMetrics_streaming_lastReceivedBatch_processingEndTime_value', 'timestamp'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for 3_NettyBlockTransfer_shuffle-server_usedDirectMemory_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for 3_executor_bytesWritten_count encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for 3_jvm_heap_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for 3_jvm_heap_used_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for 3_jvm_pools_PS-Eden-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for 3_jvm_pools_PS-Eden-Space_used_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for 3_jvm_pools_PS-Old-Gen_used_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for 3_jvm_pools_PS-Survivor-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-15') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 1_2_100000_68-15: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app1/1_2_100000_68.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/1_2_100000_68-15.test.csv\n",
      "\n",
      "## Processing 1_2_100000_68-16\n",
      "  1_2_100000_68-16 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 1_2_100000_68.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (2936, 46)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 1_0_100000_16.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (3582, 45)\n",
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_1_20_10000_100000_batch076_1__StreamingMetrics_streaming_lastReceivedBatch_processingEndTime_value', 'timestamp'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for 3_NettyBlockTransfer_shuffle-server_usedDirectMemory_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for 3_executor_bytesWritten_count encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for 3_jvm_heap_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for 3_jvm_heap_used_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for 3_jvm_pools_PS-Eden-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for 3_jvm_pools_PS-Eden-Space_used_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for 3_jvm_pools_PS-Old-Gen_used_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for 3_jvm_pools_PS-Survivor-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '1_2_100000_68-16') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 1_2_100000_68-16: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app1/1_2_100000_68.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/1_2_100000_68-16.test.csv\n",
      "\n",
      "## Processing 1_4_1000000_80-14\n",
      "  1_4_1000000_80-14 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 1_4_1000000_80.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43087, 6)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 1_0_1000000_14.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (14347, 5)\n",
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_1_18_10000_1000000_batch146_20__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
      "[('Exathlon', '1_4_1000000_80-14') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 1_4_1000000_80-14: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app1/1_4_1000000_80.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/1_4_1000000_80-14.test.csv\n",
      "\n",
      "## Processing 1_5_1000000_86-14\n",
      "  1_5_1000000_86-14 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 1_5_1000000_86.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (3618, 2)\n",
      "\n",
      "## Processing 2_1_100000_60-20\n",
      "  2_1_100000_60-20 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 2_1_100000_60.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 1_5_1000000_86-14!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46655, 16)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 2_0_100000_20.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28636, 14)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_2_20_10000_100000_batch20_52__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value', 'timestamp', 'driver_benchmark_userclicks_2_20_10000_100000_batch20_52__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '2_1_100000_60-20') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 2_1_100000_60-20: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app2/2_1_100000_60.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/2_1_100000_60-20.test.csv\n",
      "\n",
      "## Processing 2_1_100000_60-22\n",
      "  2_1_100000_60-22 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 2_1_100000_60.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46655, 16)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 2_0_100000_22.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (4257, 14)\n",
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_2_20_10000_100000_batch20_52__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value', 'timestamp', 'driver_benchmark_userclicks_2_20_10000_100000_batch20_52__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
      "[('Exathlon', '2_1_100000_60-22') (train)] KPSS trend stationarity test for driver_DAGScheduler_stage_waitingStages_value encountered an error: cannot convert float infinity to integer\n",
      "[('Exathlon', '2_1_100000_60-22') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 2_1_100000_60-22: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app2/2_1_100000_60.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/2_1_100000_60-22.test.csv\n",
      "  2_2_200000_69 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 2_2_200000_69.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (2874, 132)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 2_2_200000_69: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app2/2_2_200000_69.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/2_2_200000_69.test.csv\n",
      "\n",
      "## Processing 2_5_1000000_87-88\n",
      "  2_5_1000000_87-88 is supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 2_5_1000000_87.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43099, 2)\n",
      "\n",
      "## Processing 2_5_1000000_88-87\n",
      "  2_5_1000000_88-87 is supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 2_5_1000000_88.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 2_5_1000000_87-88!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (3621, 2)\n",
      "  3_2_500000_70 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 3_2_500000_70.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 2_5_1000000_88-87!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (2611, 210)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 3_2_500000_70: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app3/3_2_500000_70.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/3_2_500000_70.test.csv\n",
      "  3_2_1000000_71 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 3_2_1000000_71.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (2474, 196)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 3_2_1000000_71: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app3/3_2_1000000_71.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/3_2_1000000_71.test.csv\n",
      "  3_4_1000000_81 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 3_4_1000000_81.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (4214, 2)\n",
      "  3_5_1000000_89 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 3_5_1000000_89.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 3_4_1000000_81!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (5916, 2)\n",
      "\n",
      "## Processing 4_1_100000_61-27\n",
      "  4_1_100000_61-27 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 4_1_100000_61.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 3_5_1000000_89!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (129197, 22)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 4_0_100000_27.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28718, 21)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_4_20_10000_100000_batch20_50__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '4_1_100000_61-27') (train)] KPSS trend stationarity test for 4_jvm_pools_PS-Survivor-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '4_1_100000_61-27') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 4_1_100000_61-27: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app4/4_1_100000_61.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/4_1_100000_61-27.test.csv\n",
      "\n",
      "## Processing 4_1_100000_61-28\n",
      "  4_1_100000_61-28 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 4_1_100000_61.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (129197, 22)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 4_0_100000_28.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28696, 21)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_4_20_10000_100000_batch20_50__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '4_1_100000_61-28') (train)] KPSS trend stationarity test for 4_jvm_pools_PS-Survivor-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '4_1_100000_61-28') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 4_1_100000_61-28: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app4/4_1_100000_61.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/4_1_100000_61-28.test.csv\n",
      "\n",
      "## Processing 4_1_100000_61-29\n",
      "  4_1_100000_61-29 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 4_1_100000_61.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (129197, 22)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 4_0_100000_29.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28703, 21)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_4_20_10000_100000_batch20_50__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '4_1_100000_61-29') (train)] KPSS trend stationarity test for 4_jvm_pools_PS-Survivor-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '4_1_100000_61-29') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 4_1_100000_61-29: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app4/4_1_100000_61.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/4_1_100000_61-29.test.csv\n",
      "\n",
      "## Processing 4_1_100000_61-30\n",
      "  4_1_100000_61-30 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 4_1_100000_61.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (129197, 22)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 4_0_100000_30.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28707, 21)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_4_20_10000_100000_batch20_50__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '4_1_100000_61-30') (train)] KPSS trend stationarity test for 4_jvm_pools_PS-Survivor-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '4_1_100000_61-30') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 4_1_100000_61-30: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app4/4_1_100000_61.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/4_1_100000_61-30.test.csv\n",
      "\n",
      "## Processing 4_1_100000_61-32\n",
      "  4_1_100000_61-32 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 4_1_100000_61.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (129197, 22)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 4_0_100000_32.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (86182, 21)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_4_20_10000_100000_batch20_50__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '4_1_100000_61-32') (train)] KPSS trend stationarity test for 4_jvm_pools_PS-Survivor-Space_committed_value encountered an error: cannot convert float NaN to integer\n",
      "[('Exathlon', '4_1_100000_61-32') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 4_1_100000_61-32: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app4/4_1_100000_61.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/4_1_100000_61-32.test.csv\n",
      "\n",
      "## Processing 4_5_1000000_90-31\n",
      "  4_5_1000000_90-31 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 4_5_1000000_90.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (3621, 2)\n",
      "\n",
      "## Processing 5_1_100000_63-33\n",
      "  5_1_100000_63-33 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_63.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 4_5_1000000_90-31!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43066, 35)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_33.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28704, 33)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '5_1_100000_63-33') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_63-33: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_63.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_63-33.test.csv\n",
      "\n",
      "## Processing 5_1_100000_63-34\n",
      "  5_1_100000_63-34 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_63.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43066, 35)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_34.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28702, 33)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '5_1_100000_63-34') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_63-34: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_63.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_63-34.test.csv\n",
      "\n",
      "## Processing 5_1_100000_63-35\n",
      "  5_1_100000_63-35 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_63.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43066, 35)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_35.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28702, 33)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '5_1_100000_63-35') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_63-35: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_63.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_63-35.test.csv\n",
      "\n",
      "## Processing 5_1_100000_63-36\n",
      "  5_1_100000_63-36 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_63.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43066, 35)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_36.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (4724, 33)\n",
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
      "[('Exathlon', '5_1_100000_63-36') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_63-36: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_63.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_63-36.test.csv\n",
      "\n",
      "## Processing 5_1_100000_63-37\n",
      "  5_1_100000_63-37 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_63.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43066, 35)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_37.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (3581, 33)\n",
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
      "[('Exathlon', '5_1_100000_63-37') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_63-37: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_63.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_63-37.test.csv\n",
      "\n",
      "## Processing 5_1_100000_63-40\n",
      "  5_1_100000_63-40 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_63.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43066, 35)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_40.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28699, 33)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '5_1_100000_63-40') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_63-40: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_63.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_63-40.test.csv\n",
      "\n",
      "## Processing 5_1_100000_63-64\n",
      "  5_1_100000_63-64 is supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_63.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43066, 35)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_1_100000_64.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (46660, 33)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_51__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n",
      "Processed dataset 5_1_100000_63-64: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_63.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_63-64.test.csv\n",
      "\n",
      "## Processing 5_1_100000_64-33\n",
      "  5_1_100000_64-33 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_64.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46660, 23)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_33.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28704, 20)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastReceivedBatch_records_value', 'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '5_1_100000_64-33') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_64-33: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_64.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_64-33.test.csv\n",
      "\n",
      "## Processing 5_1_100000_64-34\n",
      "  5_1_100000_64-34 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_64.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46660, 23)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_34.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28702, 20)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastReceivedBatch_records_value', 'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '5_1_100000_64-34') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_64-34: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_64.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_64-34.test.csv\n",
      "\n",
      "## Processing 5_1_100000_64-35\n",
      "  5_1_100000_64-35 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_64.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46660, 23)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_35.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28702, 20)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastReceivedBatch_records_value', 'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '5_1_100000_64-35') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_64-35: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_64.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_64-35.test.csv\n",
      "\n",
      "## Processing 5_1_100000_64-36\n",
      "  5_1_100000_64-36 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_64.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46660, 23)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_36.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (4724, 20)\n",
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastReceivedBatch_records_value', 'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
      "[('Exathlon', '5_1_100000_64-36') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_64-36: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_64.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_64-36.test.csv\n",
      "\n",
      "## Processing 5_1_100000_64-37\n",
      "  5_1_100000_64-37 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_64.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46660, 23)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_37.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (3581, 20)\n",
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastReceivedBatch_records_value', 'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
      "[('Exathlon', '5_1_100000_64-37') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_64-37: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_64.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_64-37.test.csv\n",
      "\n",
      "## Processing 5_1_100000_64-40\n",
      "  5_1_100000_64-40 is semi-supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_64.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46660, 23)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_0_100000_40.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (28699, 20)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastReceivedBatch_records_value', 'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[('Exathlon', '5_1_100000_64-40') (train)] KPSS trend stationarity test for is_anomaly encountered an error: cannot convert float NaN to integer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed dataset 5_1_100000_64-40: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_64.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_64-40.test.csv\n",
      "\n",
      "## Processing 5_1_100000_64-63\n",
      "  5_1_100000_64-63 is supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_100000_64.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46660, 23)\n",
      "  Analyzing test metadata\n",
      "  Preparing train dataset\n",
      "  > reading dataset 5_1_100000_63.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > reducing columns\n",
      "  > final dataset shape: (43066, 20)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:29: UserWarning: Preselected columns contain column names that are not in the dataset ({'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastReceivedBatch_records_value', 'timestamp', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_totalDelay_value', 'driver_benchmark_userclicks_7_20_10000_100000_batch20_53__StreamingMetrics_streaming_lastCompletedBatch_processingDelay_value'})\n",
      "  warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Analyzing train metadata\n",
      "Processed dataset 5_1_100000_64-63: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_100000_64.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_100000_64-63.test.csv\n",
      "  5_1_500000_62 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_1_500000_62.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46660, 18)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 5_1_500000_62: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_1_500000_62.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_1_500000_62.test.csv\n",
      "  5_2_1000000_72 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_2_1000000_72.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (2481, 197)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 5_2_1000000_72: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app5/5_2_1000000_72.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/5_2_1000000_72.test.csv\n",
      "  5_4_1000000_82 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_4_1000000_82.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (4218, 2)\n",
      "\n",
      "## Processing 5_5_1000000_91-92\n",
      "  5_5_1000000_91-92 is supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_5_1000000_91.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 5_4_1000000_82!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43075, 2)\n",
      "\n",
      "## Processing 5_5_1000000_92-91\n",
      "  5_5_1000000_92-91 is supervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 5_5_1000000_92.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 5_5_1000000_91-92!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (3614, 2)\n",
      "  6_3_200000_76 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 6_3_200000_76.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 5_5_1000000_92-91!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46654, 9)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 6_3_200000_76: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app6/6_3_200000_76.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/6_3_200000_76.test.csv\n",
      "  6_1_500000_65 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 6_1_500000_65.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46649, 14)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 6_1_500000_65: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app6/6_1_500000_65.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/6_1_500000_65.test.csv\n",
      "  6_5_1000000_93 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 6_5_1000000_93.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (3616, 2)\n",
      "  8_3_200000_73 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 8_3_200000_73.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 6_5_1000000_93!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46641, 6)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 8_3_200000_73: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app8/8_3_200000_73.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/8_3_200000_73.test.csv\n",
      "  8_4_1000000_77 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 8_4_1000000_77.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43078, 4)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 8_4_1000000_77: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app8/8_4_1000000_77.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/8_4_1000000_77.test.csv\n",
      "  8_5_1000000_83 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 8_5_1000000_83.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (5909, 2)\n",
      "  9_3_500000_74 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 9_3_500000_74.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 8_5_1000000_83!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46650, 9)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 9_3_500000_74: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app9/9_3_500000_74.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/9_3_500000_74.test.csv\n",
      "  9_2_1000000_66 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 9_2_1000000_66.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (7481, 241)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 9_2_1000000_66: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app9/9_2_1000000_66.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/9_2_1000000_66.test.csv\n",
      "  9_4_1000000_78 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 9_4_1000000_78.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43105, 5)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 9_4_1000000_78: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app9/9_4_1000000_78.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/9_4_1000000_78.test.csv\n",
      "  9_5_1000000_84 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 9_5_1000000_84.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (5918, 2)\n",
      "  10_2_1000000_67 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 10_2_1000000_67.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 9_5_1000000_84!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (10250, 167)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 10_2_1000000_67: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app10/10_2_1000000_67.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/10_2_1000000_67.test.csv\n",
      "  10_3_1000000_75 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 10_3_1000000_75.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (46656, 10)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 10_3_1000000_75: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app10/10_3_1000000_75.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/10_3_1000000_75.test.csv\n",
      "  10_4_1000000_79 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 10_4_1000000_79.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (43086, 5)\n",
      "  Analyzing test metadata\n",
      "Processed dataset 10_4_1000000_79: /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-raw/exathlon/data/raw/app10/10_4_1000000_79.csv -> /home/sebastian/Documents/Projects/akita/data/benchmark-data/data-processed/multivariate/Exathlon/10_4_1000000_79.test.csv\n",
      "  10_5_1000000_85 is unsupervised!\n",
      "  Preparing test dataset\n",
      "  > reading dataset 10_5_1000000_85.csv\n",
      "  > adding timestamp index\n",
      "  > adding anomaly labels\n",
      "  > removing constant and empty columns\n",
      "  > selecting with label correlated columns\n",
      "  > removing highly correlated columns\n",
      "  > final dataset shape: (5910, 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_27033/1691224462.py:87: UserWarning: Found no columns that correlate with anomaly labels. Manual inspection needed for dataset 10_5_1000000_85!\n",
      "  warnings.warn(f\"Found no columns that correlate with anomaly labels. Manual inspection needed for dataset {dataset_name}!\")\n"
     ]
    }
   ],
   "source": [
    "for _, s_dataset in df_datasets[df_datasets[\"tpe\"] != 0].iterrows():\n",
    "    dataset_name = s_dataset[\"filename\"]\n",
    "    df_partners = df_datasets.loc[s_dataset.name]\n",
    "    df_partners = df_partners[(df_partners[\"filename\"] != dataset_name) & (df_partners[\"tpe\"].isin([s_dataset[\"tpe\"], 0]))]\n",
    "    \n",
    "    if df_partners.shape[0] == 0:\n",
    "        print(f\"  {dataset_name} is unsupervised!\")\n",
    "        transform_and_record(s_dataset, train_is_normal=False, train_type=\"unsupervised\")\n",
    "        continue\n",
    "\n",
    "    for _, s_partner in df_partners.iterrows():\n",
    "        dataset_name = s_dataset[\"filename\"] + \"-\" + s_partner[\"filename\"].split(\"_\")[-1]\n",
    "        print(f\"\\n## Processing {dataset_name}\")\n",
    "\n",
    "        if s_partner[\"tpe\"] == 0:\n",
    "            print(f\"  {dataset_name} is semi-supervised!\")\n",
    "            train_is_normal = True\n",
    "            train_type = \"semi-supervised\"\n",
    "        elif s_partner[\"tpe\"] == s_dataset[\"tpe\"]:\n",
    "            print(f\"  {dataset_name} is supervised!\")\n",
    "            train_is_normal = False\n",
    "            train_type = \"supervised\"\n",
    "        else:\n",
    "            print(f\"Found uncompatible partners:\\ndataset={s_dataset}\\npartner={s_partner}\")\n",
    "            continue\n",
    "\n",
    "        transform_and_record(s_dataset, s_partner, train_is_normal=train_is_normal, train_type=train_type)\n",
    "\n",
    "dm.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>train_path</th>\n",
       "      <th>test_path</th>\n",
       "      <th>dataset_type</th>\n",
       "      <th>datetime_index</th>\n",
       "      <th>split_at</th>\n",
       "      <th>train_type</th>\n",
       "      <th>train_is_normal</th>\n",
       "      <th>input_type</th>\n",
       "      <th>length</th>\n",
       "      <th>dimensions</th>\n",
       "      <th>contamination</th>\n",
       "      <th>num_anomalies</th>\n",
       "      <th>min_anomaly_length</th>\n",
       "      <th>median_anomaly_length</th>\n",
       "      <th>max_anomaly_length</th>\n",
       "      <th>mean</th>\n",
       "      <th>stddev</th>\n",
       "      <th>trend</th>\n",
       "      <th>stationarity</th>\n",
       "      <th>period_size</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>collection_name</th>\n",
       "      <th>dataset_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"39\" valign=\"top\">Exathlon</th>\n",
       "      <th>10_2_1000000_67</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/10_2_1000000_67.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>10250</td>\n",
       "      <td>165</td>\n",
       "      <td>0.510146</td>\n",
       "      <td>1</td>\n",
       "      <td>5229</td>\n",
       "      <td>5229</td>\n",
       "      <td>5229</td>\n",
       "      <td>3.475919e+09</td>\n",
       "      <td>4.479414e+09</td>\n",
       "      <td>kubic trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10_3_1000000_75</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/10_3_1000000_75.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46656</td>\n",
       "      <td>8</td>\n",
       "      <td>0.081169</td>\n",
       "      <td>4</td>\n",
       "      <td>885</td>\n",
       "      <td>965</td>\n",
       "      <td>971</td>\n",
       "      <td>6.158659e+05</td>\n",
       "      <td>1.787138e+05</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10_4_1000000_79</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/10_4_1000000_79.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43086</td>\n",
       "      <td>3</td>\n",
       "      <td>0.128766</td>\n",
       "      <td>6</td>\n",
       "      <td>474</td>\n",
       "      <td>831</td>\n",
       "      <td>1581</td>\n",
       "      <td>1.689696e+03</td>\n",
       "      <td>2.716630e+02</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_2_100000_68-15</th>\n",
       "      <td>multivariate/Exathlon/1_2_100000_68-15.train.csv</td>\n",
       "      <td>multivariate/Exathlon/1_2_100000_68-15.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>2936</td>\n",
       "      <td>44</td>\n",
       "      <td>0.788147</td>\n",
       "      <td>1</td>\n",
       "      <td>2314</td>\n",
       "      <td>2314</td>\n",
       "      <td>2314</td>\n",
       "      <td>1.169711e+10</td>\n",
       "      <td>1.685565e+10</td>\n",
       "      <td>kubic trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_2_100000_68-16</th>\n",
       "      <td>multivariate/Exathlon/1_2_100000_68-16.train.csv</td>\n",
       "      <td>multivariate/Exathlon/1_2_100000_68-16.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>2936</td>\n",
       "      <td>44</td>\n",
       "      <td>0.788147</td>\n",
       "      <td>1</td>\n",
       "      <td>2314</td>\n",
       "      <td>2314</td>\n",
       "      <td>2314</td>\n",
       "      <td>1.169711e+10</td>\n",
       "      <td>1.685565e+10</td>\n",
       "      <td>kubic trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_4_1000000_80-14</th>\n",
       "      <td>multivariate/Exathlon/1_4_1000000_80-14.train.csv</td>\n",
       "      <td>multivariate/Exathlon/1_4_1000000_80-14.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43087</td>\n",
       "      <td>4</td>\n",
       "      <td>0.128299</td>\n",
       "      <td>6</td>\n",
       "      <td>464</td>\n",
       "      <td>828</td>\n",
       "      <td>1538</td>\n",
       "      <td>7.983560e+02</td>\n",
       "      <td>1.563176e+02</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2_1_100000_60-20</th>\n",
       "      <td>multivariate/Exathlon/2_1_100000_60-20.train.csv</td>\n",
       "      <td>multivariate/Exathlon/2_1_100000_60-20.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46655</td>\n",
       "      <td>14</td>\n",
       "      <td>0.063187</td>\n",
       "      <td>3</td>\n",
       "      <td>978</td>\n",
       "      <td>982</td>\n",
       "      <td>988</td>\n",
       "      <td>1.050236e+09</td>\n",
       "      <td>1.246107e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2_1_100000_60-22</th>\n",
       "      <td>multivariate/Exathlon/2_1_100000_60-22.train.csv</td>\n",
       "      <td>multivariate/Exathlon/2_1_100000_60-22.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46655</td>\n",
       "      <td>14</td>\n",
       "      <td>0.063187</td>\n",
       "      <td>3</td>\n",
       "      <td>978</td>\n",
       "      <td>982</td>\n",
       "      <td>988</td>\n",
       "      <td>1.050236e+09</td>\n",
       "      <td>1.246107e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2_2_200000_69</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/2_2_200000_69.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>2874</td>\n",
       "      <td>130</td>\n",
       "      <td>0.575505</td>\n",
       "      <td>1</td>\n",
       "      <td>1654</td>\n",
       "      <td>1654</td>\n",
       "      <td>1654</td>\n",
       "      <td>1.888070e+10</td>\n",
       "      <td>2.136342e+10</td>\n",
       "      <td>kubic trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3_2_1000000_71</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/3_2_1000000_71.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>2474</td>\n",
       "      <td>194</td>\n",
       "      <td>0.183104</td>\n",
       "      <td>1</td>\n",
       "      <td>453</td>\n",
       "      <td>453</td>\n",
       "      <td>453</td>\n",
       "      <td>6.744530e+09</td>\n",
       "      <td>2.986543e+09</td>\n",
       "      <td>kubic trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3_2_500000_70</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/3_2_500000_70.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>2611</td>\n",
       "      <td>208</td>\n",
       "      <td>0.304864</td>\n",
       "      <td>1</td>\n",
       "      <td>796</td>\n",
       "      <td>796</td>\n",
       "      <td>796</td>\n",
       "      <td>9.331903e+09</td>\n",
       "      <td>7.031951e+09</td>\n",
       "      <td>kubic trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_100000_61-27</th>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-27.train.csv</td>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-27.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>129197</td>\n",
       "      <td>20</td>\n",
       "      <td>0.124655</td>\n",
       "      <td>9</td>\n",
       "      <td>903</td>\n",
       "      <td>1916</td>\n",
       "      <td>1961</td>\n",
       "      <td>8.511323e+08</td>\n",
       "      <td>2.113370e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_100000_61-28</th>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-28.train.csv</td>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-28.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>129197</td>\n",
       "      <td>20</td>\n",
       "      <td>0.124655</td>\n",
       "      <td>9</td>\n",
       "      <td>903</td>\n",
       "      <td>1916</td>\n",
       "      <td>1961</td>\n",
       "      <td>8.511323e+08</td>\n",
       "      <td>2.113370e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_100000_61-29</th>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-29.train.csv</td>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-29.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>129197</td>\n",
       "      <td>20</td>\n",
       "      <td>0.124655</td>\n",
       "      <td>9</td>\n",
       "      <td>903</td>\n",
       "      <td>1916</td>\n",
       "      <td>1961</td>\n",
       "      <td>8.511323e+08</td>\n",
       "      <td>2.113370e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_100000_61-30</th>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-30.train.csv</td>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-30.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>129197</td>\n",
       "      <td>20</td>\n",
       "      <td>0.124655</td>\n",
       "      <td>9</td>\n",
       "      <td>903</td>\n",
       "      <td>1916</td>\n",
       "      <td>1961</td>\n",
       "      <td>8.511323e+08</td>\n",
       "      <td>2.113370e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_1_100000_61-32</th>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-32.train.csv</td>\n",
       "      <td>multivariate/Exathlon/4_1_100000_61-32.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>129197</td>\n",
       "      <td>20</td>\n",
       "      <td>0.124655</td>\n",
       "      <td>9</td>\n",
       "      <td>903</td>\n",
       "      <td>1916</td>\n",
       "      <td>1961</td>\n",
       "      <td>8.511323e+08</td>\n",
       "      <td>2.113370e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_63-33</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-33.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-33.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43066</td>\n",
       "      <td>33</td>\n",
       "      <td>0.126248</td>\n",
       "      <td>3</td>\n",
       "      <td>1755</td>\n",
       "      <td>1838</td>\n",
       "      <td>1844</td>\n",
       "      <td>8.215836e+08</td>\n",
       "      <td>1.590597e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_63-34</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-34.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-34.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43066</td>\n",
       "      <td>33</td>\n",
       "      <td>0.126248</td>\n",
       "      <td>3</td>\n",
       "      <td>1755</td>\n",
       "      <td>1838</td>\n",
       "      <td>1844</td>\n",
       "      <td>8.215836e+08</td>\n",
       "      <td>1.590597e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_63-35</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-35.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-35.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43066</td>\n",
       "      <td>33</td>\n",
       "      <td>0.126248</td>\n",
       "      <td>3</td>\n",
       "      <td>1755</td>\n",
       "      <td>1838</td>\n",
       "      <td>1844</td>\n",
       "      <td>8.215836e+08</td>\n",
       "      <td>1.590597e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_63-36</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-36.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-36.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43066</td>\n",
       "      <td>33</td>\n",
       "      <td>0.126248</td>\n",
       "      <td>3</td>\n",
       "      <td>1755</td>\n",
       "      <td>1838</td>\n",
       "      <td>1844</td>\n",
       "      <td>8.215836e+08</td>\n",
       "      <td>1.590597e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_63-37</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-37.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-37.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43066</td>\n",
       "      <td>33</td>\n",
       "      <td>0.126248</td>\n",
       "      <td>3</td>\n",
       "      <td>1755</td>\n",
       "      <td>1838</td>\n",
       "      <td>1844</td>\n",
       "      <td>8.215836e+08</td>\n",
       "      <td>1.590597e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_63-40</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-40.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-40.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43066</td>\n",
       "      <td>33</td>\n",
       "      <td>0.126248</td>\n",
       "      <td>3</td>\n",
       "      <td>1755</td>\n",
       "      <td>1838</td>\n",
       "      <td>1844</td>\n",
       "      <td>8.215836e+08</td>\n",
       "      <td>1.590597e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_63-64</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-64.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_63-64.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43066</td>\n",
       "      <td>33</td>\n",
       "      <td>0.126248</td>\n",
       "      <td>3</td>\n",
       "      <td>1755</td>\n",
       "      <td>1838</td>\n",
       "      <td>1844</td>\n",
       "      <td>8.215836e+08</td>\n",
       "      <td>1.590597e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_64-33</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-33.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-33.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46660</td>\n",
       "      <td>21</td>\n",
       "      <td>0.062752</td>\n",
       "      <td>3</td>\n",
       "      <td>950</td>\n",
       "      <td>953</td>\n",
       "      <td>1025</td>\n",
       "      <td>7.181602e+08</td>\n",
       "      <td>8.488551e+07</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_64-34</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-34.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-34.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46660</td>\n",
       "      <td>21</td>\n",
       "      <td>0.062752</td>\n",
       "      <td>3</td>\n",
       "      <td>950</td>\n",
       "      <td>953</td>\n",
       "      <td>1025</td>\n",
       "      <td>7.181602e+08</td>\n",
       "      <td>8.488551e+07</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_64-35</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-35.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-35.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46660</td>\n",
       "      <td>21</td>\n",
       "      <td>0.062752</td>\n",
       "      <td>3</td>\n",
       "      <td>950</td>\n",
       "      <td>953</td>\n",
       "      <td>1025</td>\n",
       "      <td>7.181602e+08</td>\n",
       "      <td>8.488551e+07</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_64-36</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-36.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-36.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46660</td>\n",
       "      <td>21</td>\n",
       "      <td>0.062752</td>\n",
       "      <td>3</td>\n",
       "      <td>950</td>\n",
       "      <td>953</td>\n",
       "      <td>1025</td>\n",
       "      <td>7.181602e+08</td>\n",
       "      <td>8.488551e+07</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_64-37</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-37.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-37.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46660</td>\n",
       "      <td>21</td>\n",
       "      <td>0.062752</td>\n",
       "      <td>3</td>\n",
       "      <td>950</td>\n",
       "      <td>953</td>\n",
       "      <td>1025</td>\n",
       "      <td>7.181602e+08</td>\n",
       "      <td>8.488551e+07</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_64-40</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-40.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-40.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>semi-supervised</td>\n",
       "      <td>True</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46660</td>\n",
       "      <td>21</td>\n",
       "      <td>0.062752</td>\n",
       "      <td>3</td>\n",
       "      <td>950</td>\n",
       "      <td>953</td>\n",
       "      <td>1025</td>\n",
       "      <td>7.181602e+08</td>\n",
       "      <td>8.488551e+07</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_100000_64-63</th>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-63.train.csv</td>\n",
       "      <td>multivariate/Exathlon/5_1_100000_64-63.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46660</td>\n",
       "      <td>21</td>\n",
       "      <td>0.062752</td>\n",
       "      <td>3</td>\n",
       "      <td>950</td>\n",
       "      <td>953</td>\n",
       "      <td>1025</td>\n",
       "      <td>7.181602e+08</td>\n",
       "      <td>8.488551e+07</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_1_500000_62</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/5_1_500000_62.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46660</td>\n",
       "      <td>16</td>\n",
       "      <td>0.124668</td>\n",
       "      <td>6</td>\n",
       "      <td>964</td>\n",
       "      <td>969</td>\n",
       "      <td>980</td>\n",
       "      <td>9.242092e+10</td>\n",
       "      <td>2.041997e+10</td>\n",
       "      <td>no trend</td>\n",
       "      <td>not_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_2_1000000_72</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/5_2_1000000_72.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>2481</td>\n",
       "      <td>195</td>\n",
       "      <td>0.384925</td>\n",
       "      <td>1</td>\n",
       "      <td>955</td>\n",
       "      <td>955</td>\n",
       "      <td>955</td>\n",
       "      <td>7.533925e+09</td>\n",
       "      <td>7.503620e+09</td>\n",
       "      <td>kubic trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6_1_500000_65</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/6_1_500000_65.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46649</td>\n",
       "      <td>12</td>\n",
       "      <td>0.104097</td>\n",
       "      <td>5</td>\n",
       "      <td>964</td>\n",
       "      <td>969</td>\n",
       "      <td>979</td>\n",
       "      <td>3.118153e+09</td>\n",
       "      <td>2.751184e+08</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6_3_200000_76</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/6_3_200000_76.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46654</td>\n",
       "      <td>7</td>\n",
       "      <td>0.080165</td>\n",
       "      <td>4</td>\n",
       "      <td>855</td>\n",
       "      <td>961</td>\n",
       "      <td>962</td>\n",
       "      <td>3.617637e+07</td>\n",
       "      <td>9.862455e+06</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_3_200000_73</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/8_3_200000_73.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46641</td>\n",
       "      <td>4</td>\n",
       "      <td>0.081002</td>\n",
       "      <td>4</td>\n",
       "      <td>865</td>\n",
       "      <td>969</td>\n",
       "      <td>975</td>\n",
       "      <td>2.306513e+05</td>\n",
       "      <td>7.218720e+04</td>\n",
       "      <td>no trend</td>\n",
       "      <td>difference_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8_4_1000000_77</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/8_4_1000000_77.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43078</td>\n",
       "      <td>2</td>\n",
       "      <td>0.130624</td>\n",
       "      <td>6</td>\n",
       "      <td>467</td>\n",
       "      <td>821</td>\n",
       "      <td>1563</td>\n",
       "      <td>4.156744e+01</td>\n",
       "      <td>1.619839e+01</td>\n",
       "      <td>no trend</td>\n",
       "      <td>stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9_2_1000000_66</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/9_2_1000000_66.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>7481</td>\n",
       "      <td>239</td>\n",
       "      <td>0.460233</td>\n",
       "      <td>1</td>\n",
       "      <td>3443</td>\n",
       "      <td>3443</td>\n",
       "      <td>3443</td>\n",
       "      <td>2.388802e+09</td>\n",
       "      <td>3.099851e+09</td>\n",
       "      <td>kubic trend</td>\n",
       "      <td>trend_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9_3_500000_74</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/9_3_500000_74.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>46650</td>\n",
       "      <td>7</td>\n",
       "      <td>0.083408</td>\n",
       "      <td>4</td>\n",
       "      <td>972</td>\n",
       "      <td>973</td>\n",
       "      <td>973</td>\n",
       "      <td>6.706471e+05</td>\n",
       "      <td>1.967543e+05</td>\n",
       "      <td>no trend</td>\n",
       "      <td>not_stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9_4_1000000_78</th>\n",
       "      <td>NaN</td>\n",
       "      <td>multivariate/Exathlon/9_4_1000000_78.test.csv</td>\n",
       "      <td>real</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unsupervised</td>\n",
       "      <td>False</td>\n",
       "      <td>multivariate</td>\n",
       "      <td>43105</td>\n",
       "      <td>3</td>\n",
       "      <td>0.129962</td>\n",
       "      <td>6</td>\n",
       "      <td>481</td>\n",
       "      <td>853</td>\n",
       "      <td>1637</td>\n",
       "      <td>2.327022e+03</td>\n",
       "      <td>5.612242e+02</td>\n",
       "      <td>no trend</td>\n",
       "      <td>stationary</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                          train_path  \\\n",
       "collection_name dataset_name                                                           \n",
       "Exathlon        10_2_1000000_67                                                  NaN   \n",
       "                10_3_1000000_75                                                  NaN   \n",
       "                10_4_1000000_79                                                  NaN   \n",
       "                1_2_100000_68-15    multivariate/Exathlon/1_2_100000_68-15.train.csv   \n",
       "                1_2_100000_68-16    multivariate/Exathlon/1_2_100000_68-16.train.csv   \n",
       "                1_4_1000000_80-14  multivariate/Exathlon/1_4_1000000_80-14.train.csv   \n",
       "                2_1_100000_60-20    multivariate/Exathlon/2_1_100000_60-20.train.csv   \n",
       "                2_1_100000_60-22    multivariate/Exathlon/2_1_100000_60-22.train.csv   \n",
       "                2_2_200000_69                                                    NaN   \n",
       "                3_2_1000000_71                                                   NaN   \n",
       "                3_2_500000_70                                                    NaN   \n",
       "                4_1_100000_61-27    multivariate/Exathlon/4_1_100000_61-27.train.csv   \n",
       "                4_1_100000_61-28    multivariate/Exathlon/4_1_100000_61-28.train.csv   \n",
       "                4_1_100000_61-29    multivariate/Exathlon/4_1_100000_61-29.train.csv   \n",
       "                4_1_100000_61-30    multivariate/Exathlon/4_1_100000_61-30.train.csv   \n",
       "                4_1_100000_61-32    multivariate/Exathlon/4_1_100000_61-32.train.csv   \n",
       "                5_1_100000_63-33    multivariate/Exathlon/5_1_100000_63-33.train.csv   \n",
       "                5_1_100000_63-34    multivariate/Exathlon/5_1_100000_63-34.train.csv   \n",
       "                5_1_100000_63-35    multivariate/Exathlon/5_1_100000_63-35.train.csv   \n",
       "                5_1_100000_63-36    multivariate/Exathlon/5_1_100000_63-36.train.csv   \n",
       "                5_1_100000_63-37    multivariate/Exathlon/5_1_100000_63-37.train.csv   \n",
       "                5_1_100000_63-40    multivariate/Exathlon/5_1_100000_63-40.train.csv   \n",
       "                5_1_100000_63-64    multivariate/Exathlon/5_1_100000_63-64.train.csv   \n",
       "                5_1_100000_64-33    multivariate/Exathlon/5_1_100000_64-33.train.csv   \n",
       "                5_1_100000_64-34    multivariate/Exathlon/5_1_100000_64-34.train.csv   \n",
       "                5_1_100000_64-35    multivariate/Exathlon/5_1_100000_64-35.train.csv   \n",
       "                5_1_100000_64-36    multivariate/Exathlon/5_1_100000_64-36.train.csv   \n",
       "                5_1_100000_64-37    multivariate/Exathlon/5_1_100000_64-37.train.csv   \n",
       "                5_1_100000_64-40    multivariate/Exathlon/5_1_100000_64-40.train.csv   \n",
       "                5_1_100000_64-63    multivariate/Exathlon/5_1_100000_64-63.train.csv   \n",
       "                5_1_500000_62                                                    NaN   \n",
       "                5_2_1000000_72                                                   NaN   \n",
       "                6_1_500000_65                                                    NaN   \n",
       "                6_3_200000_76                                                    NaN   \n",
       "                8_3_200000_73                                                    NaN   \n",
       "                8_4_1000000_77                                                   NaN   \n",
       "                9_2_1000000_66                                                   NaN   \n",
       "                9_3_500000_74                                                    NaN   \n",
       "                9_4_1000000_78                                                   NaN   \n",
       "\n",
       "                                                                          test_path  \\\n",
       "collection_name dataset_name                                                          \n",
       "Exathlon        10_2_1000000_67      multivariate/Exathlon/10_2_1000000_67.test.csv   \n",
       "                10_3_1000000_75      multivariate/Exathlon/10_3_1000000_75.test.csv   \n",
       "                10_4_1000000_79      multivariate/Exathlon/10_4_1000000_79.test.csv   \n",
       "                1_2_100000_68-15    multivariate/Exathlon/1_2_100000_68-15.test.csv   \n",
       "                1_2_100000_68-16    multivariate/Exathlon/1_2_100000_68-16.test.csv   \n",
       "                1_4_1000000_80-14  multivariate/Exathlon/1_4_1000000_80-14.test.csv   \n",
       "                2_1_100000_60-20    multivariate/Exathlon/2_1_100000_60-20.test.csv   \n",
       "                2_1_100000_60-22    multivariate/Exathlon/2_1_100000_60-22.test.csv   \n",
       "                2_2_200000_69          multivariate/Exathlon/2_2_200000_69.test.csv   \n",
       "                3_2_1000000_71        multivariate/Exathlon/3_2_1000000_71.test.csv   \n",
       "                3_2_500000_70          multivariate/Exathlon/3_2_500000_70.test.csv   \n",
       "                4_1_100000_61-27    multivariate/Exathlon/4_1_100000_61-27.test.csv   \n",
       "                4_1_100000_61-28    multivariate/Exathlon/4_1_100000_61-28.test.csv   \n",
       "                4_1_100000_61-29    multivariate/Exathlon/4_1_100000_61-29.test.csv   \n",
       "                4_1_100000_61-30    multivariate/Exathlon/4_1_100000_61-30.test.csv   \n",
       "                4_1_100000_61-32    multivariate/Exathlon/4_1_100000_61-32.test.csv   \n",
       "                5_1_100000_63-33    multivariate/Exathlon/5_1_100000_63-33.test.csv   \n",
       "                5_1_100000_63-34    multivariate/Exathlon/5_1_100000_63-34.test.csv   \n",
       "                5_1_100000_63-35    multivariate/Exathlon/5_1_100000_63-35.test.csv   \n",
       "                5_1_100000_63-36    multivariate/Exathlon/5_1_100000_63-36.test.csv   \n",
       "                5_1_100000_63-37    multivariate/Exathlon/5_1_100000_63-37.test.csv   \n",
       "                5_1_100000_63-40    multivariate/Exathlon/5_1_100000_63-40.test.csv   \n",
       "                5_1_100000_63-64    multivariate/Exathlon/5_1_100000_63-64.test.csv   \n",
       "                5_1_100000_64-33    multivariate/Exathlon/5_1_100000_64-33.test.csv   \n",
       "                5_1_100000_64-34    multivariate/Exathlon/5_1_100000_64-34.test.csv   \n",
       "                5_1_100000_64-35    multivariate/Exathlon/5_1_100000_64-35.test.csv   \n",
       "                5_1_100000_64-36    multivariate/Exathlon/5_1_100000_64-36.test.csv   \n",
       "                5_1_100000_64-37    multivariate/Exathlon/5_1_100000_64-37.test.csv   \n",
       "                5_1_100000_64-40    multivariate/Exathlon/5_1_100000_64-40.test.csv   \n",
       "                5_1_100000_64-63    multivariate/Exathlon/5_1_100000_64-63.test.csv   \n",
       "                5_1_500000_62          multivariate/Exathlon/5_1_500000_62.test.csv   \n",
       "                5_2_1000000_72        multivariate/Exathlon/5_2_1000000_72.test.csv   \n",
       "                6_1_500000_65          multivariate/Exathlon/6_1_500000_65.test.csv   \n",
       "                6_3_200000_76          multivariate/Exathlon/6_3_200000_76.test.csv   \n",
       "                8_3_200000_73          multivariate/Exathlon/8_3_200000_73.test.csv   \n",
       "                8_4_1000000_77        multivariate/Exathlon/8_4_1000000_77.test.csv   \n",
       "                9_2_1000000_66        multivariate/Exathlon/9_2_1000000_66.test.csv   \n",
       "                9_3_500000_74          multivariate/Exathlon/9_3_500000_74.test.csv   \n",
       "                9_4_1000000_78        multivariate/Exathlon/9_4_1000000_78.test.csv   \n",
       "\n",
       "                                  dataset_type  datetime_index  split_at  \\\n",
       "collection_name dataset_name                                               \n",
       "Exathlon        10_2_1000000_67           real            True       NaN   \n",
       "                10_3_1000000_75           real            True       NaN   \n",
       "                10_4_1000000_79           real            True       NaN   \n",
       "                1_2_100000_68-15          real            True       NaN   \n",
       "                1_2_100000_68-16          real            True       NaN   \n",
       "                1_4_1000000_80-14         real            True       NaN   \n",
       "                2_1_100000_60-20          real            True       NaN   \n",
       "                2_1_100000_60-22          real            True       NaN   \n",
       "                2_2_200000_69             real            True       NaN   \n",
       "                3_2_1000000_71            real            True       NaN   \n",
       "                3_2_500000_70             real            True       NaN   \n",
       "                4_1_100000_61-27          real            True       NaN   \n",
       "                4_1_100000_61-28          real            True       NaN   \n",
       "                4_1_100000_61-29          real            True       NaN   \n",
       "                4_1_100000_61-30          real            True       NaN   \n",
       "                4_1_100000_61-32          real            True       NaN   \n",
       "                5_1_100000_63-33          real            True       NaN   \n",
       "                5_1_100000_63-34          real            True       NaN   \n",
       "                5_1_100000_63-35          real            True       NaN   \n",
       "                5_1_100000_63-36          real            True       NaN   \n",
       "                5_1_100000_63-37          real            True       NaN   \n",
       "                5_1_100000_63-40          real            True       NaN   \n",
       "                5_1_100000_63-64          real            True       NaN   \n",
       "                5_1_100000_64-33          real            True       NaN   \n",
       "                5_1_100000_64-34          real            True       NaN   \n",
       "                5_1_100000_64-35          real            True       NaN   \n",
       "                5_1_100000_64-36          real            True       NaN   \n",
       "                5_1_100000_64-37          real            True       NaN   \n",
       "                5_1_100000_64-40          real            True       NaN   \n",
       "                5_1_100000_64-63          real            True       NaN   \n",
       "                5_1_500000_62             real            True       NaN   \n",
       "                5_2_1000000_72            real            True       NaN   \n",
       "                6_1_500000_65             real            True       NaN   \n",
       "                6_3_200000_76             real            True       NaN   \n",
       "                8_3_200000_73             real            True       NaN   \n",
       "                8_4_1000000_77            real            True       NaN   \n",
       "                9_2_1000000_66            real            True       NaN   \n",
       "                9_3_500000_74             real            True       NaN   \n",
       "                9_4_1000000_78            real            True       NaN   \n",
       "\n",
       "                                        train_type  train_is_normal  \\\n",
       "collection_name dataset_name                                          \n",
       "Exathlon        10_2_1000000_67       unsupervised            False   \n",
       "                10_3_1000000_75       unsupervised            False   \n",
       "                10_4_1000000_79       unsupervised            False   \n",
       "                1_2_100000_68-15   semi-supervised             True   \n",
       "                1_2_100000_68-16   semi-supervised             True   \n",
       "                1_4_1000000_80-14  semi-supervised             True   \n",
       "                2_1_100000_60-20   semi-supervised             True   \n",
       "                2_1_100000_60-22   semi-supervised             True   \n",
       "                2_2_200000_69         unsupervised            False   \n",
       "                3_2_1000000_71        unsupervised            False   \n",
       "                3_2_500000_70         unsupervised            False   \n",
       "                4_1_100000_61-27   semi-supervised             True   \n",
       "                4_1_100000_61-28   semi-supervised             True   \n",
       "                4_1_100000_61-29   semi-supervised             True   \n",
       "                4_1_100000_61-30   semi-supervised             True   \n",
       "                4_1_100000_61-32   semi-supervised             True   \n",
       "                5_1_100000_63-33   semi-supervised             True   \n",
       "                5_1_100000_63-34   semi-supervised             True   \n",
       "                5_1_100000_63-35   semi-supervised             True   \n",
       "                5_1_100000_63-36   semi-supervised             True   \n",
       "                5_1_100000_63-37   semi-supervised             True   \n",
       "                5_1_100000_63-40   semi-supervised             True   \n",
       "                5_1_100000_63-64        supervised            False   \n",
       "                5_1_100000_64-33   semi-supervised             True   \n",
       "                5_1_100000_64-34   semi-supervised             True   \n",
       "                5_1_100000_64-35   semi-supervised             True   \n",
       "                5_1_100000_64-36   semi-supervised             True   \n",
       "                5_1_100000_64-37   semi-supervised             True   \n",
       "                5_1_100000_64-40   semi-supervised             True   \n",
       "                5_1_100000_64-63        supervised            False   \n",
       "                5_1_500000_62         unsupervised            False   \n",
       "                5_2_1000000_72        unsupervised            False   \n",
       "                6_1_500000_65         unsupervised            False   \n",
       "                6_3_200000_76         unsupervised            False   \n",
       "                8_3_200000_73         unsupervised            False   \n",
       "                8_4_1000000_77        unsupervised            False   \n",
       "                9_2_1000000_66        unsupervised            False   \n",
       "                9_3_500000_74         unsupervised            False   \n",
       "                9_4_1000000_78        unsupervised            False   \n",
       "\n",
       "                                     input_type  length  dimensions  \\\n",
       "collection_name dataset_name                                          \n",
       "Exathlon        10_2_1000000_67    multivariate   10250         165   \n",
       "                10_3_1000000_75    multivariate   46656           8   \n",
       "                10_4_1000000_79    multivariate   43086           3   \n",
       "                1_2_100000_68-15   multivariate    2936          44   \n",
       "                1_2_100000_68-16   multivariate    2936          44   \n",
       "                1_4_1000000_80-14  multivariate   43087           4   \n",
       "                2_1_100000_60-20   multivariate   46655          14   \n",
       "                2_1_100000_60-22   multivariate   46655          14   \n",
       "                2_2_200000_69      multivariate    2874         130   \n",
       "                3_2_1000000_71     multivariate    2474         194   \n",
       "                3_2_500000_70      multivariate    2611         208   \n",
       "                4_1_100000_61-27   multivariate  129197          20   \n",
       "                4_1_100000_61-28   multivariate  129197          20   \n",
       "                4_1_100000_61-29   multivariate  129197          20   \n",
       "                4_1_100000_61-30   multivariate  129197          20   \n",
       "                4_1_100000_61-32   multivariate  129197          20   \n",
       "                5_1_100000_63-33   multivariate   43066          33   \n",
       "                5_1_100000_63-34   multivariate   43066          33   \n",
       "                5_1_100000_63-35   multivariate   43066          33   \n",
       "                5_1_100000_63-36   multivariate   43066          33   \n",
       "                5_1_100000_63-37   multivariate   43066          33   \n",
       "                5_1_100000_63-40   multivariate   43066          33   \n",
       "                5_1_100000_63-64   multivariate   43066          33   \n",
       "                5_1_100000_64-33   multivariate   46660          21   \n",
       "                5_1_100000_64-34   multivariate   46660          21   \n",
       "                5_1_100000_64-35   multivariate   46660          21   \n",
       "                5_1_100000_64-36   multivariate   46660          21   \n",
       "                5_1_100000_64-37   multivariate   46660          21   \n",
       "                5_1_100000_64-40   multivariate   46660          21   \n",
       "                5_1_100000_64-63   multivariate   46660          21   \n",
       "                5_1_500000_62      multivariate   46660          16   \n",
       "                5_2_1000000_72     multivariate    2481         195   \n",
       "                6_1_500000_65      multivariate   46649          12   \n",
       "                6_3_200000_76      multivariate   46654           7   \n",
       "                8_3_200000_73      multivariate   46641           4   \n",
       "                8_4_1000000_77     multivariate   43078           2   \n",
       "                9_2_1000000_66     multivariate    7481         239   \n",
       "                9_3_500000_74      multivariate   46650           7   \n",
       "                9_4_1000000_78     multivariate   43105           3   \n",
       "\n",
       "                                   contamination  num_anomalies  \\\n",
       "collection_name dataset_name                                      \n",
       "Exathlon        10_2_1000000_67         0.510146              1   \n",
       "                10_3_1000000_75         0.081169              4   \n",
       "                10_4_1000000_79         0.128766              6   \n",
       "                1_2_100000_68-15        0.788147              1   \n",
       "                1_2_100000_68-16        0.788147              1   \n",
       "                1_4_1000000_80-14       0.128299              6   \n",
       "                2_1_100000_60-20        0.063187              3   \n",
       "                2_1_100000_60-22        0.063187              3   \n",
       "                2_2_200000_69           0.575505              1   \n",
       "                3_2_1000000_71          0.183104              1   \n",
       "                3_2_500000_70           0.304864              1   \n",
       "                4_1_100000_61-27        0.124655              9   \n",
       "                4_1_100000_61-28        0.124655              9   \n",
       "                4_1_100000_61-29        0.124655              9   \n",
       "                4_1_100000_61-30        0.124655              9   \n",
       "                4_1_100000_61-32        0.124655              9   \n",
       "                5_1_100000_63-33        0.126248              3   \n",
       "                5_1_100000_63-34        0.126248              3   \n",
       "                5_1_100000_63-35        0.126248              3   \n",
       "                5_1_100000_63-36        0.126248              3   \n",
       "                5_1_100000_63-37        0.126248              3   \n",
       "                5_1_100000_63-40        0.126248              3   \n",
       "                5_1_100000_63-64        0.126248              3   \n",
       "                5_1_100000_64-33        0.062752              3   \n",
       "                5_1_100000_64-34        0.062752              3   \n",
       "                5_1_100000_64-35        0.062752              3   \n",
       "                5_1_100000_64-36        0.062752              3   \n",
       "                5_1_100000_64-37        0.062752              3   \n",
       "                5_1_100000_64-40        0.062752              3   \n",
       "                5_1_100000_64-63        0.062752              3   \n",
       "                5_1_500000_62           0.124668              6   \n",
       "                5_2_1000000_72          0.384925              1   \n",
       "                6_1_500000_65           0.104097              5   \n",
       "                6_3_200000_76           0.080165              4   \n",
       "                8_3_200000_73           0.081002              4   \n",
       "                8_4_1000000_77          0.130624              6   \n",
       "                9_2_1000000_66          0.460233              1   \n",
       "                9_3_500000_74           0.083408              4   \n",
       "                9_4_1000000_78          0.129962              6   \n",
       "\n",
       "                                   min_anomaly_length  median_anomaly_length  \\\n",
       "collection_name dataset_name                                                   \n",
       "Exathlon        10_2_1000000_67                  5229                   5229   \n",
       "                10_3_1000000_75                   885                    965   \n",
       "                10_4_1000000_79                   474                    831   \n",
       "                1_2_100000_68-15                 2314                   2314   \n",
       "                1_2_100000_68-16                 2314                   2314   \n",
       "                1_4_1000000_80-14                 464                    828   \n",
       "                2_1_100000_60-20                  978                    982   \n",
       "                2_1_100000_60-22                  978                    982   \n",
       "                2_2_200000_69                    1654                   1654   \n",
       "                3_2_1000000_71                    453                    453   \n",
       "                3_2_500000_70                     796                    796   \n",
       "                4_1_100000_61-27                  903                   1916   \n",
       "                4_1_100000_61-28                  903                   1916   \n",
       "                4_1_100000_61-29                  903                   1916   \n",
       "                4_1_100000_61-30                  903                   1916   \n",
       "                4_1_100000_61-32                  903                   1916   \n",
       "                5_1_100000_63-33                 1755                   1838   \n",
       "                5_1_100000_63-34                 1755                   1838   \n",
       "                5_1_100000_63-35                 1755                   1838   \n",
       "                5_1_100000_63-36                 1755                   1838   \n",
       "                5_1_100000_63-37                 1755                   1838   \n",
       "                5_1_100000_63-40                 1755                   1838   \n",
       "                5_1_100000_63-64                 1755                   1838   \n",
       "                5_1_100000_64-33                  950                    953   \n",
       "                5_1_100000_64-34                  950                    953   \n",
       "                5_1_100000_64-35                  950                    953   \n",
       "                5_1_100000_64-36                  950                    953   \n",
       "                5_1_100000_64-37                  950                    953   \n",
       "                5_1_100000_64-40                  950                    953   \n",
       "                5_1_100000_64-63                  950                    953   \n",
       "                5_1_500000_62                     964                    969   \n",
       "                5_2_1000000_72                    955                    955   \n",
       "                6_1_500000_65                     964                    969   \n",
       "                6_3_200000_76                     855                    961   \n",
       "                8_3_200000_73                     865                    969   \n",
       "                8_4_1000000_77                    467                    821   \n",
       "                9_2_1000000_66                   3443                   3443   \n",
       "                9_3_500000_74                     972                    973   \n",
       "                9_4_1000000_78                    481                    853   \n",
       "\n",
       "                                   max_anomaly_length          mean  \\\n",
       "collection_name dataset_name                                          \n",
       "Exathlon        10_2_1000000_67                  5229  3.475919e+09   \n",
       "                10_3_1000000_75                   971  6.158659e+05   \n",
       "                10_4_1000000_79                  1581  1.689696e+03   \n",
       "                1_2_100000_68-15                 2314  1.169711e+10   \n",
       "                1_2_100000_68-16                 2314  1.169711e+10   \n",
       "                1_4_1000000_80-14                1538  7.983560e+02   \n",
       "                2_1_100000_60-20                  988  1.050236e+09   \n",
       "                2_1_100000_60-22                  988  1.050236e+09   \n",
       "                2_2_200000_69                    1654  1.888070e+10   \n",
       "                3_2_1000000_71                    453  6.744530e+09   \n",
       "                3_2_500000_70                     796  9.331903e+09   \n",
       "                4_1_100000_61-27                 1961  8.511323e+08   \n",
       "                4_1_100000_61-28                 1961  8.511323e+08   \n",
       "                4_1_100000_61-29                 1961  8.511323e+08   \n",
       "                4_1_100000_61-30                 1961  8.511323e+08   \n",
       "                4_1_100000_61-32                 1961  8.511323e+08   \n",
       "                5_1_100000_63-33                 1844  8.215836e+08   \n",
       "                5_1_100000_63-34                 1844  8.215836e+08   \n",
       "                5_1_100000_63-35                 1844  8.215836e+08   \n",
       "                5_1_100000_63-36                 1844  8.215836e+08   \n",
       "                5_1_100000_63-37                 1844  8.215836e+08   \n",
       "                5_1_100000_63-40                 1844  8.215836e+08   \n",
       "                5_1_100000_63-64                 1844  8.215836e+08   \n",
       "                5_1_100000_64-33                 1025  7.181602e+08   \n",
       "                5_1_100000_64-34                 1025  7.181602e+08   \n",
       "                5_1_100000_64-35                 1025  7.181602e+08   \n",
       "                5_1_100000_64-36                 1025  7.181602e+08   \n",
       "                5_1_100000_64-37                 1025  7.181602e+08   \n",
       "                5_1_100000_64-40                 1025  7.181602e+08   \n",
       "                5_1_100000_64-63                 1025  7.181602e+08   \n",
       "                5_1_500000_62                     980  9.242092e+10   \n",
       "                5_2_1000000_72                    955  7.533925e+09   \n",
       "                6_1_500000_65                     979  3.118153e+09   \n",
       "                6_3_200000_76                     962  3.617637e+07   \n",
       "                8_3_200000_73                     975  2.306513e+05   \n",
       "                8_4_1000000_77                   1563  4.156744e+01   \n",
       "                9_2_1000000_66                   3443  2.388802e+09   \n",
       "                9_3_500000_74                     973  6.706471e+05   \n",
       "                9_4_1000000_78                   1637  2.327022e+03   \n",
       "\n",
       "                                         stddev        trend  \\\n",
       "collection_name dataset_name                                   \n",
       "Exathlon        10_2_1000000_67    4.479414e+09  kubic trend   \n",
       "                10_3_1000000_75    1.787138e+05     no trend   \n",
       "                10_4_1000000_79    2.716630e+02     no trend   \n",
       "                1_2_100000_68-15   1.685565e+10  kubic trend   \n",
       "                1_2_100000_68-16   1.685565e+10  kubic trend   \n",
       "                1_4_1000000_80-14  1.563176e+02     no trend   \n",
       "                2_1_100000_60-20   1.246107e+08     no trend   \n",
       "                2_1_100000_60-22   1.246107e+08     no trend   \n",
       "                2_2_200000_69      2.136342e+10  kubic trend   \n",
       "                3_2_1000000_71     2.986543e+09  kubic trend   \n",
       "                3_2_500000_70      7.031951e+09  kubic trend   \n",
       "                4_1_100000_61-27   2.113370e+08     no trend   \n",
       "                4_1_100000_61-28   2.113370e+08     no trend   \n",
       "                4_1_100000_61-29   2.113370e+08     no trend   \n",
       "                4_1_100000_61-30   2.113370e+08     no trend   \n",
       "                4_1_100000_61-32   2.113370e+08     no trend   \n",
       "                5_1_100000_63-33   1.590597e+08     no trend   \n",
       "                5_1_100000_63-34   1.590597e+08     no trend   \n",
       "                5_1_100000_63-35   1.590597e+08     no trend   \n",
       "                5_1_100000_63-36   1.590597e+08     no trend   \n",
       "                5_1_100000_63-37   1.590597e+08     no trend   \n",
       "                5_1_100000_63-40   1.590597e+08     no trend   \n",
       "                5_1_100000_63-64   1.590597e+08     no trend   \n",
       "                5_1_100000_64-33   8.488551e+07     no trend   \n",
       "                5_1_100000_64-34   8.488551e+07     no trend   \n",
       "                5_1_100000_64-35   8.488551e+07     no trend   \n",
       "                5_1_100000_64-36   8.488551e+07     no trend   \n",
       "                5_1_100000_64-37   8.488551e+07     no trend   \n",
       "                5_1_100000_64-40   8.488551e+07     no trend   \n",
       "                5_1_100000_64-63   8.488551e+07     no trend   \n",
       "                5_1_500000_62      2.041997e+10     no trend   \n",
       "                5_2_1000000_72     7.503620e+09  kubic trend   \n",
       "                6_1_500000_65      2.751184e+08     no trend   \n",
       "                6_3_200000_76      9.862455e+06     no trend   \n",
       "                8_3_200000_73      7.218720e+04     no trend   \n",
       "                8_4_1000000_77     1.619839e+01     no trend   \n",
       "                9_2_1000000_66     3.099851e+09  kubic trend   \n",
       "                9_3_500000_74      1.967543e+05     no trend   \n",
       "                9_4_1000000_78     5.612242e+02     no trend   \n",
       "\n",
       "                                            stationarity  period_size  \n",
       "collection_name dataset_name                                           \n",
       "Exathlon        10_2_1000000_67         trend_stationary          NaN  \n",
       "                10_3_1000000_75    difference_stationary          NaN  \n",
       "                10_4_1000000_79    difference_stationary          NaN  \n",
       "                1_2_100000_68-15        trend_stationary          NaN  \n",
       "                1_2_100000_68-16        trend_stationary          NaN  \n",
       "                1_4_1000000_80-14  difference_stationary          NaN  \n",
       "                2_1_100000_60-20        trend_stationary          NaN  \n",
       "                2_1_100000_60-22        trend_stationary          NaN  \n",
       "                2_2_200000_69           trend_stationary          NaN  \n",
       "                3_2_1000000_71          trend_stationary          NaN  \n",
       "                3_2_500000_70           trend_stationary          NaN  \n",
       "                4_1_100000_61-27   difference_stationary          NaN  \n",
       "                4_1_100000_61-28   difference_stationary          NaN  \n",
       "                4_1_100000_61-29   difference_stationary          NaN  \n",
       "                4_1_100000_61-30   difference_stationary          NaN  \n",
       "                4_1_100000_61-32   difference_stationary          NaN  \n",
       "                5_1_100000_63-33        trend_stationary          NaN  \n",
       "                5_1_100000_63-34        trend_stationary          NaN  \n",
       "                5_1_100000_63-35        trend_stationary          NaN  \n",
       "                5_1_100000_63-36        trend_stationary          NaN  \n",
       "                5_1_100000_63-37        trend_stationary          NaN  \n",
       "                5_1_100000_63-40        trend_stationary          NaN  \n",
       "                5_1_100000_63-64        trend_stationary          NaN  \n",
       "                5_1_100000_64-33   difference_stationary          NaN  \n",
       "                5_1_100000_64-34   difference_stationary          NaN  \n",
       "                5_1_100000_64-35   difference_stationary          NaN  \n",
       "                5_1_100000_64-36   difference_stationary          NaN  \n",
       "                5_1_100000_64-37   difference_stationary          NaN  \n",
       "                5_1_100000_64-40   difference_stationary          NaN  \n",
       "                5_1_100000_64-63   difference_stationary          NaN  \n",
       "                5_1_500000_62             not_stationary          NaN  \n",
       "                5_2_1000000_72          trend_stationary          NaN  \n",
       "                6_1_500000_65      difference_stationary          NaN  \n",
       "                6_3_200000_76      difference_stationary          NaN  \n",
       "                8_3_200000_73      difference_stationary          NaN  \n",
       "                8_4_1000000_77                stationary          NaN  \n",
       "                9_2_1000000_66          trend_stationary          NaN  \n",
       "                9_3_500000_74             not_stationary          NaN  \n",
       "                9_4_1000000_78                stationary          NaN  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dm.refresh()\n",
    "dm.df().loc[(slice(dataset_collection_name,dataset_collection_name), slice(None))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Datasets for which our automatic extraction of relevant columns (dimensions/channels/variates) did not work properly:\n",
    "\n",
    "- 1_5_1000000_86\n",
    "- 2_5_1000000_87\n",
    "- 2_5_1000000_88\n",
    "- 3_4_1000000_81\n",
    "- 3_5_1000000_89\n",
    "- 4_5_1000000_90\n",
    "- 5_4_1000000_82\n",
    "- 5_5_1000000_91\n",
    "- 5_5_1000000_92\n",
    "- 6_5_1000000_93\n",
    "- 8_5_1000000_83\n",
    "- 9_5_1000000_84\n",
    "- 10_5_1000000_85\n",
    "\n",
    "These datasets require manual inspection and preprocessing!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "plot_dataset(datasets[list(datasets.keys())[23]], 110, 140)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(datasets[23])\n",
    "print(f\"Columns = {df.shape[1]}\")\n",
    "print(f\"Rows = {df.shape[0]}\")\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[c for c in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop highly correlated columns\n",
    "df_corr = df.corr().abs()\n",
    "df_corr = df_corr.where(np.triu(np.ones(df_corr.shape), k=1).astype(np.bool_))\n",
    "df_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 0.95\n",
    "to_drop = [c for c in df_corr.columns if any(df_corr[c] > threshold)]\n",
    "df2 = df.drop(columns=to_drop)\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "datasets = find_datasets(source_folder)\n",
    "datasets = dict((d.stem, d) for d in datasets)\n",
    "df_datasets = pd.DataFrame({\"filename\": list(datasets.keys())}, columns=[\"filename\", \"app\", \"tpe\", \"rate\"])\n",
    "df_datasets.iloc[:, 1:] = df_datasets[\"filename\"].str.split(\"_\", expand=True).iloc[:, :3]\n",
    "df_datasets.loc[:, [\"app\", \"tpe\", \"rate\"]] = df_datasets[[\"app\", \"tpe\", \"rate\"]].astype(np.int_)\n",
    "res = df_datasets.groupby(by=[\"app\", \"rate\", \"tpe\"]).count()\n",
    "df_datasets = df_datasets.set_index([\"app\", \"rate\"]).sort_index()\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(dataset_name: Path, corr_threshold: float = 0.95, log_prefix: str = \"  \", preselected_columns: Optional[List[str]] = None):\n",
    "    print(f\"{log_prefix}reading dataset {dataset_name.name}\")\n",
    "    df = pd.read_csv(dataset_name)\n",
    "    s_index = pd.Index(pd.to_datetime(df[\"t\"], unit=\"s\"), name=\"timestamp\")\n",
    "    df.drop(columns=[\"t\"], inplace=True)\n",
    "    \n",
    "    print(f\"{log_prefix}adding timestamp index\")\n",
    "    df.index = s_index\n",
    "        \n",
    "    # add labels\n",
    "    print(f\"{log_prefix}adding anomaly labels\")\n",
    "    df[\"is_anomaly\"] = 0\n",
    "    try:\n",
    "        anomalies = df_labels.loc[dataset_name.stem:dataset_name.stem]\n",
    "    except KeyError:\n",
    "        anomalies = pd.DataFrame(columns=df_labels.columns, index=pd.Index([], name=\"trace\"))\n",
    "    for i, anomaly in anomalies.iterrows():\n",
    "        begin = pd.to_datetime(anomaly[\"begin\"], unit=\"s\")\n",
    "        end = pd.to_datetime(anomaly[\"end\"], unit=\"s\")\n",
    "        df.loc[begin:end, \"is_anomaly\"] = 1\n",
    "    \n",
    "    if preselected_columns is not None:\n",
    "        # use columns from test dataset (preselected)\n",
    "        print(f\"{log_prefix}reducing columns\")\n",
    "        columns = list(np.unique(list(preselected_columns) + [\"is_anomaly\"]))\n",
    "        unavailable_columns = set(columns) - set(df.columns)\n",
    "        if len(unavailable_columns) > 0:\n",
    "            warnings.warn(f\"Preselected columns contain column names that are not in the dataset ({unavailable_columns})\")\n",
    "        df = df[[c for c in columns if c not in unavailable_columns]]\n",
    "    else:\n",
    "        # columns with no real value\n",
    "        print(f\"{log_prefix}removing constant and empty columns\")\n",
    "        to_drop = [c for c in df.columns if c != \"is_anomaly\" and (np.all(df[c].isin([0, -1])) or df[c].unique().shape[0] == 1)]\n",
    "        df.drop(columns=to_drop, inplace=True)\n",
    "\n",
    "        # only select correlated values\n",
    "        print(f\"{log_prefix}selecting with label correlated columns\")\n",
    "        s_corr = df.drop(columns=[\"is_anomaly\"]).corrwith(df[\"is_anomaly\"], drop=True).abs()\n",
    "        cols = list(s_corr[s_corr > 0.5].index.values)\n",
    "        df = df[cols + [\"is_anomaly\"]]\n",
    "\n",
    "        # drop highly correlated columns\n",
    "        print(f\"{log_prefix}removing highly correlated columns\")\n",
    "        df_corr = df.drop(columns=[\"is_anomaly\"]).corr().abs()\n",
    "        df_corr = df_corr.where(np.triu(np.ones(df_corr.shape), k=1).astype(np.bool_))\n",
    "        to_drop = [c for c in df_corr.columns if any(df_corr[c] > corr_threshold)]\n",
    "        df.drop(columns=to_drop, inplace=True)\n",
    "        \n",
    "    # make timestamp index a column\n",
    "    df.reset_index(drop=False, inplace=True)\n",
    "    print(f\"{log_prefix}final dataset shape: {df.shape}\")\n",
    "\n",
    "    return df\n",
    "    \n",
    "\n",
    "df = preprocess(datasets[list(datasets.keys())[24]])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_corr = df.drop(columns=[\"is_anomaly\"]).corrwith(df[\"is_anomaly\"], drop=True).abs()\n",
    "corr_cols = df_corr[df_corr > 0.5]#.index.values\n",
    "corr_cols.sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[[\"driver_BlockManager_memory_memUsed_MB_value\"]].plot()\n",
    "#plt.gca().set_ylim(0, 20)\n",
    "#plt.gca().set_xlim(pd.to_datetime(1.527534e+09, unit=\"s\"), pd.to_datetime(1.527537e+09, unit=\"s\"))\n",
    "plt.gca().get_legend().remove()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "timeeval",
   "language": "python",
   "name": "timeeval"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
